{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cifar-10-python.tar.gz']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input\"))\n",
    "\n",
    "import time\n",
    "\n",
    "# import pytorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import SGD,Adam,lr_scheduler\n",
    "from torch.utils.data import random_split\n",
    "import torchvision\n",
    "from torchvision import transforms, datasets\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define transformations for train\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(p=.40),\n",
    "    transforms.RandomRotation(30),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define transformations for test\n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define training dataloader\n",
    "def get_training_dataloader(train_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        train_transform: transfroms for train dataset\n",
    "        path: path to cifar100 training python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: train_data_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_train = train_transform\n",
    "    cifar10_training = torchvision.datasets.CIFAR10(root='.', train=True, download=True, transform=transform_train)\n",
    "    cifar10_training_loader = DataLoader(\n",
    "        cifar10_training, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_training_loader\n",
    "\n",
    "# define test dataloader\n",
    "def get_testing_dataloader(test_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        test_transform: transforms for test dataset\n",
    "        path: path to cifar100 test python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: cifar100_test_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_test = test_transform\n",
    "    cifar10_test = torchvision.datasets.CIFAR10(root='.', train=False, download=True, transform=transform_test)\n",
    "    cifar10_test_loader = DataLoader(\n",
    "        cifar10_test, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement mish activation function\n",
    "def f_mish(input):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "    '''\n",
    "    return input * torch.tanh(F.softplus(input))\n",
    "\n",
    "# implement class wrapper for mish activation function\n",
    "class mish(nn.Module):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = mish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_mish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement swish activation function\n",
    "def f_swish(input):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "    '''\n",
    "    return input * torch.sigmoid(input)\n",
    "\n",
    "# implement class wrapper for swish activation function\n",
    "class swish(nn.Module):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = swish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_swish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\"\"\"This strategy exposes a new dimension, which we call “cardinality” \n",
    "#(the size of the set of transformations), as an essential factor \n",
    "#in addition to the dimensions of depth and width.\"\"\"\n",
    "CARDINALITY = 32\n",
    "DEPTH = 4\n",
    "BASEWIDTH = 64\n",
    "\n",
    "#\"\"\"The grouped convolutional layer in Fig. 3(c) performs 32 groups \n",
    "#of convolutions whose input and output channels are 4-dimensional. \n",
    "#The grouped convolutional layer concatenates them as the outputs \n",
    "#of the layer.\"\"\"\n",
    "\n",
    "class ResNextBottleNeckC(nn.Module):\n",
    "\n",
    "    def __init__(self, in_channels, out_channels, stride, activation = 'relu'):\n",
    "        super().__init__()\n",
    "\n",
    "        C = CARDINALITY #How many groups a feature map was splitted into\n",
    "        \n",
    "        self.activation = activation\n",
    "        \n",
    "        if self.activation == 'relu':\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            \n",
    "        if self.activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            \n",
    "        if self.activation == 'mish':\n",
    "            f_activation = mish()\n",
    "\n",
    "        #\"\"\"We note that the input/output width of the template is fixed as \n",
    "        #256-d (Fig. 3), We note that the input/output width of the template \n",
    "        #is fixed as 256-d (Fig. 3), and all widths are dou- bled each time \n",
    "        #when the feature map is subsampled (see Table 1).\"\"\"\n",
    "        D = int(DEPTH * out_channels / BASEWIDTH) #number of channels per group\n",
    "        self.split_transforms = nn.Sequential(\n",
    "            nn.Conv2d(in_channels, C * D, kernel_size=1, groups=C, bias=False),\n",
    "            nn.BatchNorm2d(C * D),\n",
    "            f_activation,\n",
    "            nn.Conv2d(C * D, C * D, kernel_size=3, stride=stride, groups=C, padding=1, bias=False),\n",
    "            nn.BatchNorm2d(C * D),\n",
    "            f_activation,\n",
    "            nn.Conv2d(C * D, out_channels * 4, kernel_size=1, bias=False),\n",
    "            nn.BatchNorm2d(out_channels * 4),\n",
    "        )\n",
    "\n",
    "        self.shortcut = nn.Sequential()\n",
    "\n",
    "        if stride != 1 or in_channels != out_channels * 4:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                nn.Conv2d(in_channels, out_channels * 4, stride=stride, kernel_size=1, bias=False),\n",
    "                nn.BatchNorm2d(out_channels * 4)\n",
    "            )\n",
    "\n",
    "    def forward(self, x):\n",
    "        if self.activation == 'relu':\n",
    "            return F.relu(self.split_transforms(x) + self.shortcut(x))\n",
    "        if self.activation == 'swish':\n",
    "            return f_swish(self.split_transforms(x) + self.shortcut(x))\n",
    "        if self.activation == 'mish':\n",
    "            return f_mish(self.split_transforms(x) + self.shortcut(x))\n",
    "\n",
    "class ResNext(nn.Module):\n",
    "\n",
    "    def __init__(self, block, num_blocks, class_names=100, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        self.in_channels = 64\n",
    "        \n",
    "        self.activation = activation\n",
    "        \n",
    "        if self.activation == 'relu':\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            \n",
    "        if self.activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            \n",
    "        if self.activation == 'mish':\n",
    "            f_activation = mish()\n",
    "\n",
    "        self.conv1 = nn.Sequential(\n",
    "            nn.Conv2d(3, 64, 3, stride=1, padding=1, bias=False),\n",
    "            nn.BatchNorm2d(64),\n",
    "            f_activation\n",
    "        )\n",
    "\n",
    "        self.conv2 = self._make_layer(block, num_blocks[0], 64, 1, activation = activation)\n",
    "        self.conv3 = self._make_layer(block, num_blocks[1], 128, 2, activation = activation)\n",
    "        self.conv4 = self._make_layer(block, num_blocks[2], 256, 2, activation = activation)\n",
    "        self.conv5 = self._make_layer(block, num_blocks[3], 512, 2, activation = activation)\n",
    "        self.avg = nn.AdaptiveAvgPool2d((1, 1))\n",
    "        self.fc = nn.Linear(512 * 4, 100)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = self.conv2(x)\n",
    "        x = self.conv3(x)\n",
    "        x = self.conv4(x)\n",
    "        x = self.conv5(x)\n",
    "        x = self.avg(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "        \n",
    "    def _make_layer(self, block, num_block, out_channels, stride, activation = 'relu'):\n",
    "        \"\"\"Building resnext block\n",
    "        Args:\n",
    "            block: block type(default resnext bottleneck c)\n",
    "            num_block: number of blocks per layer\n",
    "            out_channels: output channels per block\n",
    "            stride: block stride\n",
    "        \n",
    "        Returns:\n",
    "            a resnext layer\n",
    "        \"\"\"\n",
    "        strides = [stride] + [1] * (num_block - 1)\n",
    "        layers = []\n",
    "        for stride in strides:\n",
    "            layers.append(block(self.in_channels, out_channels, stride, activation = activation))\n",
    "            self.in_channels = out_channels * 4\n",
    "\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "def resnext50(activation = 'relu'):\n",
    "    \"\"\" return a resnext50(c32x4d) network\n",
    "    \"\"\"\n",
    "    return ResNext(ResNextBottleNeckC, [3, 4, 6, 3], activation = activation)\n",
    "\n",
    "def resnext101(activation = 'relu'):\n",
    "    \"\"\" return a resnext101(c32x4d) network\n",
    "    \"\"\"\n",
    "    return ResNext(ResNextBottleNeckC, [3, 4, 23, 3], activation = activation)\n",
    "\n",
    "def resnext152(activation = 'relu'):\n",
    "    \"\"\" return a resnext101(c32x4d) network\n",
    "    \"\"\"\n",
    "    return ResNext(ResNextBottleNeckC, [3, 4, 36, 3], activation = activation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "170500096it [00:06, 27159765.76it/s]                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainloader = get_training_dataloader(train_transform)\n",
    "testloader = get_testing_dataloader(test_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs = 100\n",
    "batch_size = 128\n",
    "learning_rate = 0.001\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = resnext50(activation = 'mish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# set optimizer, only train the classifier parameters, feature parameters are frozen\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats = pd.DataFrame(columns = ['Epoch', 'Time per epoch', 'Avg time per step', 'Train loss', 'Train accuracy', 'Train top-3 accuracy','Test loss', 'Test accuracy', 'Test top-3 accuracy']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100.. Time per epoch: 179.0172.. Average time per step: 0.4578.. Train loss: 1.3613.. Train accuracy: 0.5115.. Top-3 train accuracy: 0.8185.. Test loss: 1.3132.. Test accuracy: 0.5728.. Top-3 test accuracy: 0.8602\n",
      "Epoch 2/100.. Time per epoch: 179.0110.. Average time per step: 0.4578.. Train loss: 0.9189.. Train accuracy: 0.6757.. Top-3 train accuracy: 0.9101.. Test loss: 0.8582.. Test accuracy: 0.7125.. Top-3 test accuracy: 0.9295\n",
      "Epoch 3/100.. Time per epoch: 178.7258.. Average time per step: 0.4571.. Train loss: 0.7387.. Train accuracy: 0.7438.. Top-3 train accuracy: 0.9391.. Test loss: 0.6527.. Test accuracy: 0.7786.. Top-3 test accuracy: 0.9521\n",
      "Epoch 4/100.. Time per epoch: 178.8238.. Average time per step: 0.4573.. Train loss: 0.6296.. Train accuracy: 0.7818.. Top-3 train accuracy: 0.9537.. Test loss: 0.5475.. Test accuracy: 0.8185.. Top-3 test accuracy: 0.9631\n",
      "Epoch 5/100.. Time per epoch: 178.4227.. Average time per step: 0.4563.. Train loss: 0.5530.. Train accuracy: 0.8086.. Top-3 train accuracy: 0.9620.. Test loss: 0.4887.. Test accuracy: 0.8340.. Top-3 test accuracy: 0.9705\n",
      "Epoch 6/100.. Time per epoch: 178.6085.. Average time per step: 0.4568.. Train loss: 0.4950.. Train accuracy: 0.8272.. Top-3 train accuracy: 0.9677.. Test loss: 0.4531.. Test accuracy: 0.8480.. Top-3 test accuracy: 0.9746\n",
      "Epoch 7/100.. Time per epoch: 178.1217.. Average time per step: 0.4556.. Train loss: 0.4498.. Train accuracy: 0.8439.. Top-3 train accuracy: 0.9726.. Test loss: 0.4326.. Test accuracy: 0.8553.. Top-3 test accuracy: 0.9759\n",
      "Epoch 8/100.. Time per epoch: 178.1532.. Average time per step: 0.4556.. Train loss: 0.4124.. Train accuracy: 0.8570.. Top-3 train accuracy: 0.9752.. Test loss: 0.4265.. Test accuracy: 0.8574.. Top-3 test accuracy: 0.9764\n",
      "Epoch 9/100.. Time per epoch: 178.1657.. Average time per step: 0.4557.. Train loss: 0.3794.. Train accuracy: 0.8672.. Top-3 train accuracy: 0.9797.. Test loss: 0.3800.. Test accuracy: 0.8723.. Top-3 test accuracy: 0.9778\n",
      "Epoch 10/100.. Time per epoch: 178.7513.. Average time per step: 0.4572.. Train loss: 0.3507.. Train accuracy: 0.8773.. Top-3 train accuracy: 0.9823.. Test loss: 0.3966.. Test accuracy: 0.8659.. Top-3 test accuracy: 0.9788\n",
      "Epoch 11/100.. Time per epoch: 178.5391.. Average time per step: 0.4566.. Train loss: 0.3282.. Train accuracy: 0.8858.. Top-3 train accuracy: 0.9841.. Test loss: 0.3774.. Test accuracy: 0.8771.. Top-3 test accuracy: 0.9802\n",
      "Epoch 12/100.. Time per epoch: 178.6981.. Average time per step: 0.4570.. Train loss: 0.3032.. Train accuracy: 0.8950.. Top-3 train accuracy: 0.9861.. Test loss: 0.3694.. Test accuracy: 0.8772.. Top-3 test accuracy: 0.9816\n",
      "Epoch 13/100.. Time per epoch: 178.0878.. Average time per step: 0.4555.. Train loss: 0.2820.. Train accuracy: 0.9008.. Top-3 train accuracy: 0.9875.. Test loss: 0.3578.. Test accuracy: 0.8803.. Top-3 test accuracy: 0.9822\n",
      "Epoch 14/100.. Time per epoch: 178.1629.. Average time per step: 0.4557.. Train loss: 0.2633.. Train accuracy: 0.9084.. Top-3 train accuracy: 0.9898.. Test loss: 0.3423.. Test accuracy: 0.8876.. Top-3 test accuracy: 0.9845\n",
      "Epoch 15/100.. Time per epoch: 177.8963.. Average time per step: 0.4550.. Train loss: 0.2492.. Train accuracy: 0.9129.. Top-3 train accuracy: 0.9898.. Test loss: 0.3353.. Test accuracy: 0.8908.. Top-3 test accuracy: 0.9849\n",
      "Epoch 16/100.. Time per epoch: 178.1632.. Average time per step: 0.4557.. Train loss: 0.2319.. Train accuracy: 0.9198.. Top-3 train accuracy: 0.9911.. Test loss: 0.3347.. Test accuracy: 0.8911.. Top-3 test accuracy: 0.9847\n",
      "Epoch 17/100.. Time per epoch: 177.7677.. Average time per step: 0.4546.. Train loss: 0.2199.. Train accuracy: 0.9216.. Top-3 train accuracy: 0.9918.. Test loss: 0.3558.. Test accuracy: 0.8898.. Top-3 test accuracy: 0.9829\n",
      "Epoch 18/100.. Time per epoch: 177.9804.. Average time per step: 0.4552.. Train loss: 0.2043.. Train accuracy: 0.9276.. Top-3 train accuracy: 0.9927.. Test loss: 0.3432.. Test accuracy: 0.8941.. Top-3 test accuracy: 0.9856\n",
      "Epoch 19/100.. Time per epoch: 177.4036.. Average time per step: 0.4537.. Train loss: 0.1959.. Train accuracy: 0.9299.. Top-3 train accuracy: 0.9938.. Test loss: 0.3386.. Test accuracy: 0.8934.. Top-3 test accuracy: 0.9845\n",
      "Epoch 20/100.. Time per epoch: 177.6486.. Average time per step: 0.4543.. Train loss: 0.1832.. Train accuracy: 0.9362.. Top-3 train accuracy: 0.9938.. Test loss: 0.3542.. Test accuracy: 0.8925.. Top-3 test accuracy: 0.9851\n",
      "Epoch 21/100.. Time per epoch: 178.0188.. Average time per step: 0.4553.. Train loss: 0.1746.. Train accuracy: 0.9394.. Top-3 train accuracy: 0.9949.. Test loss: 0.3319.. Test accuracy: 0.8974.. Top-3 test accuracy: 0.9858\n",
      "Epoch 22/100.. Time per epoch: 178.6424.. Average time per step: 0.4569.. Train loss: 0.1596.. Train accuracy: 0.9440.. Top-3 train accuracy: 0.9951.. Test loss: 0.3407.. Test accuracy: 0.8994.. Top-3 test accuracy: 0.9855\n",
      "Epoch 23/100.. Time per epoch: 178.5359.. Average time per step: 0.4566.. Train loss: 0.1570.. Train accuracy: 0.9453.. Top-3 train accuracy: 0.9959.. Test loss: 0.3278.. Test accuracy: 0.9001.. Top-3 test accuracy: 0.9875\n",
      "Epoch 24/100.. Time per epoch: 178.5710.. Average time per step: 0.4567.. Train loss: 0.1494.. Train accuracy: 0.9477.. Top-3 train accuracy: 0.9958.. Test loss: 0.3316.. Test accuracy: 0.9004.. Top-3 test accuracy: 0.9868\n",
      "Epoch 25/100.. Time per epoch: 178.0462.. Average time per step: 0.4554.. Train loss: 0.1411.. Train accuracy: 0.9507.. Top-3 train accuracy: 0.9963.. Test loss: 0.3353.. Test accuracy: 0.9058.. Top-3 test accuracy: 0.9854\n",
      "Epoch 26/100.. Time per epoch: 178.0455.. Average time per step: 0.4554.. Train loss: 0.1301.. Train accuracy: 0.9546.. Top-3 train accuracy: 0.9973.. Test loss: 0.3296.. Test accuracy: 0.9050.. Top-3 test accuracy: 0.9873\n",
      "Epoch 27/100.. Time per epoch: 177.6474.. Average time per step: 0.4543.. Train loss: 0.1235.. Train accuracy: 0.9566.. Top-3 train accuracy: 0.9970.. Test loss: 0.3761.. Test accuracy: 0.8970.. Top-3 test accuracy: 0.9857\n",
      "Epoch 28/100.. Time per epoch: 177.6303.. Average time per step: 0.4543.. Train loss: 0.1199.. Train accuracy: 0.9575.. Top-3 train accuracy: 0.9976.. Test loss: 0.3550.. Test accuracy: 0.9005.. Top-3 test accuracy: 0.9855\n",
      "Epoch 29/100.. Time per epoch: 177.3202.. Average time per step: 0.4535.. Train loss: 0.1207.. Train accuracy: 0.9570.. Top-3 train accuracy: 0.9973.. Test loss: 0.3497.. Test accuracy: 0.9080.. Top-3 test accuracy: 0.9866\n",
      "Epoch 30/100.. Time per epoch: 177.4320.. Average time per step: 0.4538.. Train loss: 0.1088.. Train accuracy: 0.9611.. Top-3 train accuracy: 0.9980.. Test loss: 0.3475.. Test accuracy: 0.8994.. Top-3 test accuracy: 0.9870\n",
      "Epoch 31/100.. Time per epoch: 177.1574.. Average time per step: 0.4531.. Train loss: 0.1033.. Train accuracy: 0.9638.. Top-3 train accuracy: 0.9979.. Test loss: 0.3489.. Test accuracy: 0.9100.. Top-3 test accuracy: 0.9870\n",
      "Epoch 32/100.. Time per epoch: 177.5975.. Average time per step: 0.4542.. Train loss: 0.1035.. Train accuracy: 0.9643.. Top-3 train accuracy: 0.9980.. Test loss: 0.3496.. Test accuracy: 0.9025.. Top-3 test accuracy: 0.9847\n",
      "Epoch 33/100.. Time per epoch: 176.9766.. Average time per step: 0.4526.. Train loss: 0.0998.. Train accuracy: 0.9640.. Top-3 train accuracy: 0.9980.. Test loss: 0.3685.. Test accuracy: 0.9079.. Top-3 test accuracy: 0.9858\n",
      "Epoch 34/100.. Time per epoch: 177.1715.. Average time per step: 0.4531.. Train loss: 0.0977.. Train accuracy: 0.9662.. Top-3 train accuracy: 0.9978.. Test loss: 0.3612.. Test accuracy: 0.9023.. Top-3 test accuracy: 0.9858\n",
      "Epoch 35/100.. Time per epoch: 177.1375.. Average time per step: 0.4530.. Train loss: 0.0907.. Train accuracy: 0.9688.. Top-3 train accuracy: 0.9982.. Test loss: 0.3592.. Test accuracy: 0.9079.. Top-3 test accuracy: 0.9860\n",
      "Epoch 36/100.. Time per epoch: 178.0528.. Average time per step: 0.4554.. Train loss: 0.0899.. Train accuracy: 0.9678.. Top-3 train accuracy: 0.9983.. Test loss: 0.3808.. Test accuracy: 0.9026.. Top-3 test accuracy: 0.9842\n",
      "Epoch 37/100.. Time per epoch: 178.1675.. Average time per step: 0.4557.. Train loss: 0.0873.. Train accuracy: 0.9696.. Top-3 train accuracy: 0.9984.. Test loss: 0.3529.. Test accuracy: 0.9115.. Top-3 test accuracy: 0.9865\n",
      "Epoch 38/100.. Time per epoch: 178.5677.. Average time per step: 0.4567.. Train loss: 0.0837.. Train accuracy: 0.9706.. Top-3 train accuracy: 0.9987.. Test loss: 0.3536.. Test accuracy: 0.9115.. Top-3 test accuracy: 0.9869\n",
      "Epoch 39/100.. Time per epoch: 178.0619.. Average time per step: 0.4554.. Train loss: 0.0789.. Train accuracy: 0.9724.. Top-3 train accuracy: 0.9988.. Test loss: 0.3723.. Test accuracy: 0.9054.. Top-3 test accuracy: 0.9860\n",
      "Epoch 40/100.. Time per epoch: 178.3322.. Average time per step: 0.4561.. Train loss: 0.0782.. Train accuracy: 0.9721.. Top-3 train accuracy: 0.9988.. Test loss: 0.3560.. Test accuracy: 0.9095.. Top-3 test accuracy: 0.9866\n",
      "Epoch 41/100.. Time per epoch: 177.8931.. Average time per step: 0.4550.. Train loss: 0.0714.. Train accuracy: 0.9749.. Top-3 train accuracy: 0.9991.. Test loss: 0.3637.. Test accuracy: 0.9083.. Top-3 test accuracy: 0.9871\n",
      "Epoch 42/100.. Time per epoch: 177.8100.. Average time per step: 0.4548.. Train loss: 0.0744.. Train accuracy: 0.9745.. Top-3 train accuracy: 0.9990.. Test loss: 0.3871.. Test accuracy: 0.9034.. Top-3 test accuracy: 0.9865\n",
      "Epoch 43/100.. Time per epoch: 177.3038.. Average time per step: 0.4535.. Train loss: 0.0708.. Train accuracy: 0.9754.. Top-3 train accuracy: 0.9992.. Test loss: 0.3504.. Test accuracy: 0.9120.. Top-3 test accuracy: 0.9867\n",
      "Epoch 44/100.. Time per epoch: 177.7948.. Average time per step: 0.4547.. Train loss: 0.0716.. Train accuracy: 0.9758.. Top-3 train accuracy: 0.9988.. Test loss: 0.3686.. Test accuracy: 0.9081.. Top-3 test accuracy: 0.9869\n",
      "Epoch 45/100.. Time per epoch: 177.4081.. Average time per step: 0.4537.. Train loss: 0.0660.. Train accuracy: 0.9768.. Top-3 train accuracy: 0.9991.. Test loss: 0.3837.. Test accuracy: 0.9093.. Top-3 test accuracy: 0.9850\n",
      "Epoch 46/100.. Time per epoch: 177.5030.. Average time per step: 0.4540.. Train loss: 0.0637.. Train accuracy: 0.9777.. Top-3 train accuracy: 0.9992.. Test loss: 0.3803.. Test accuracy: 0.9120.. Top-3 test accuracy: 0.9882\n",
      "Epoch 47/100.. Time per epoch: 177.4397.. Average time per step: 0.4538.. Train loss: 0.0599.. Train accuracy: 0.9794.. Top-3 train accuracy: 0.9993.. Test loss: 0.3815.. Test accuracy: 0.9103.. Top-3 test accuracy: 0.9876\n",
      "Epoch 48/100.. Time per epoch: 177.3901.. Average time per step: 0.4537.. Train loss: 0.0674.. Train accuracy: 0.9771.. Top-3 train accuracy: 0.9993.. Test loss: 0.3988.. Test accuracy: 0.9057.. Top-3 test accuracy: 0.9846\n",
      "Epoch 49/100.. Time per epoch: 177.0950.. Average time per step: 0.4529.. Train loss: 0.0619.. Train accuracy: 0.9787.. Top-3 train accuracy: 0.9991.. Test loss: 0.3655.. Test accuracy: 0.9120.. Top-3 test accuracy: 0.9868\n",
      "Epoch 50/100.. Time per epoch: 177.3534.. Average time per step: 0.4536.. Train loss: 0.0580.. Train accuracy: 0.9799.. Top-3 train accuracy: 0.9993.. Test loss: 0.3859.. Test accuracy: 0.9098.. Top-3 test accuracy: 0.9871\n",
      "Epoch 51/100.. Time per epoch: 177.0318.. Average time per step: 0.4528.. Train loss: 0.0585.. Train accuracy: 0.9795.. Top-3 train accuracy: 0.9994.. Test loss: 0.4005.. Test accuracy: 0.9083.. Top-3 test accuracy: 0.9864\n",
      "Epoch 52/100.. Time per epoch: 177.2913.. Average time per step: 0.4534.. Train loss: 0.0544.. Train accuracy: 0.9806.. Top-3 train accuracy: 0.9995.. Test loss: 0.3852.. Test accuracy: 0.9133.. Top-3 test accuracy: 0.9874\n",
      "Epoch 53/100.. Time per epoch: 176.9825.. Average time per step: 0.4526.. Train loss: 0.0560.. Train accuracy: 0.9801.. Top-3 train accuracy: 0.9995.. Test loss: 0.4171.. Test accuracy: 0.9072.. Top-3 test accuracy: 0.9858\n",
      "Epoch 54/100.. Time per epoch: 177.2079.. Average time per step: 0.4532.. Train loss: 0.0545.. Train accuracy: 0.9810.. Top-3 train accuracy: 0.9994.. Test loss: 0.3915.. Test accuracy: 0.9104.. Top-3 test accuracy: 0.9876\n",
      "Epoch 55/100.. Time per epoch: 177.0353.. Average time per step: 0.4528.. Train loss: 0.0539.. Train accuracy: 0.9813.. Top-3 train accuracy: 0.9996.. Test loss: 0.3722.. Test accuracy: 0.9130.. Top-3 test accuracy: 0.9862\n",
      "Epoch 56/100.. Time per epoch: 177.2712.. Average time per step: 0.4534.. Train loss: 0.0477.. Train accuracy: 0.9833.. Top-3 train accuracy: 0.9994.. Test loss: 0.4062.. Test accuracy: 0.9098.. Top-3 test accuracy: 0.9860\n",
      "Epoch 57/100.. Time per epoch: 177.0934.. Average time per step: 0.4529.. Train loss: 0.0513.. Train accuracy: 0.9822.. Top-3 train accuracy: 0.9995.. Test loss: 0.3819.. Test accuracy: 0.9126.. Top-3 test accuracy: 0.9887\n",
      "Epoch 58/100.. Time per epoch: 177.9572.. Average time per step: 0.4551.. Train loss: 0.0489.. Train accuracy: 0.9826.. Top-3 train accuracy: 0.9996.. Test loss: 0.4050.. Test accuracy: 0.9132.. Top-3 test accuracy: 0.9860\n",
      "Epoch 59/100.. Time per epoch: 177.6430.. Average time per step: 0.4543.. Train loss: 0.0477.. Train accuracy: 0.9840.. Top-3 train accuracy: 0.9996.. Test loss: 0.4236.. Test accuracy: 0.9090.. Top-3 test accuracy: 0.9864\n",
      "Epoch 60/100.. Time per epoch: 177.2636.. Average time per step: 0.4534.. Train loss: 0.0488.. Train accuracy: 0.9836.. Top-3 train accuracy: 0.9995.. Test loss: 0.4257.. Test accuracy: 0.9086.. Top-3 test accuracy: 0.9865\n",
      "Epoch 61/100.. Time per epoch: 176.8984.. Average time per step: 0.4524.. Train loss: 0.0471.. Train accuracy: 0.9835.. Top-3 train accuracy: 0.9995.. Test loss: 0.3721.. Test accuracy: 0.9147.. Top-3 test accuracy: 0.9878\n",
      "Epoch 62/100.. Time per epoch: 177.0572.. Average time per step: 0.4528.. Train loss: 0.0412.. Train accuracy: 0.9856.. Top-3 train accuracy: 0.9997.. Test loss: 0.4063.. Test accuracy: 0.9082.. Top-3 test accuracy: 0.9873\n",
      "Epoch 63/100.. Time per epoch: 176.9660.. Average time per step: 0.4526.. Train loss: 0.0471.. Train accuracy: 0.9841.. Top-3 train accuracy: 0.9996.. Test loss: 0.3933.. Test accuracy: 0.9115.. Top-3 test accuracy: 0.9875\n",
      "Epoch 64/100.. Time per epoch: 177.5846.. Average time per step: 0.4542.. Train loss: 0.0471.. Train accuracy: 0.9838.. Top-3 train accuracy: 0.9997.. Test loss: 0.3523.. Test accuracy: 0.9221.. Top-3 test accuracy: 0.9892\n",
      "Epoch 65/100.. Time per epoch: 177.4020.. Average time per step: 0.4537.. Train loss: 0.0444.. Train accuracy: 0.9851.. Top-3 train accuracy: 0.9995.. Test loss: 0.3815.. Test accuracy: 0.9111.. Top-3 test accuracy: 0.9866\n",
      "Epoch 66/100.. Time per epoch: 178.0168.. Average time per step: 0.4553.. Train loss: 0.0409.. Train accuracy: 0.9860.. Top-3 train accuracy: 0.9997.. Test loss: 0.4166.. Test accuracy: 0.9064.. Top-3 test accuracy: 0.9872\n",
      "Epoch 67/100.. Time per epoch: 177.1178.. Average time per step: 0.4530.. Train loss: 0.0397.. Train accuracy: 0.9862.. Top-3 train accuracy: 0.9997.. Test loss: 0.4257.. Test accuracy: 0.9070.. Top-3 test accuracy: 0.9863\n",
      "Epoch 68/100.. Time per epoch: 177.4401.. Average time per step: 0.4538.. Train loss: 0.0465.. Train accuracy: 0.9846.. Top-3 train accuracy: 0.9997.. Test loss: 0.3986.. Test accuracy: 0.9100.. Top-3 test accuracy: 0.9867\n",
      "Epoch 69/100.. Time per epoch: 177.8024.. Average time per step: 0.4547.. Train loss: 0.0369.. Train accuracy: 0.9870.. Top-3 train accuracy: 0.9997.. Test loss: 0.3887.. Test accuracy: 0.9170.. Top-3 test accuracy: 0.9867\n",
      "Epoch 70/100.. Time per epoch: 178.0131.. Average time per step: 0.4553.. Train loss: 0.0356.. Train accuracy: 0.9875.. Top-3 train accuracy: 0.9997.. Test loss: 0.4127.. Test accuracy: 0.9115.. Top-3 test accuracy: 0.9858\n",
      "Epoch 71/100.. Time per epoch: 177.9813.. Average time per step: 0.4552.. Train loss: 0.0382.. Train accuracy: 0.9867.. Top-3 train accuracy: 0.9997.. Test loss: 0.4049.. Test accuracy: 0.9105.. Top-3 test accuracy: 0.9864\n",
      "Epoch 72/100.. Time per epoch: 177.8584.. Average time per step: 0.4549.. Train loss: 0.0377.. Train accuracy: 0.9867.. Top-3 train accuracy: 0.9997.. Test loss: 0.4202.. Test accuracy: 0.9127.. Top-3 test accuracy: 0.9866\n",
      "Epoch 73/100.. Time per epoch: 177.7880.. Average time per step: 0.4547.. Train loss: 0.0403.. Train accuracy: 0.9857.. Top-3 train accuracy: 0.9996.. Test loss: 0.4335.. Test accuracy: 0.9095.. Top-3 test accuracy: 0.9850\n",
      "Epoch 74/100.. Time per epoch: 178.1396.. Average time per step: 0.4556.. Train loss: 0.0356.. Train accuracy: 0.9875.. Top-3 train accuracy: 0.9999.. Test loss: 0.4309.. Test accuracy: 0.9110.. Top-3 test accuracy: 0.9850\n",
      "Epoch 75/100.. Time per epoch: 178.0640.. Average time per step: 0.4554.. Train loss: 0.0377.. Train accuracy: 0.9861.. Top-3 train accuracy: 0.9997.. Test loss: 0.4305.. Test accuracy: 0.9068.. Top-3 test accuracy: 0.9849\n",
      "Epoch 76/100.. Time per epoch: 178.7459.. Average time per step: 0.4572.. Train loss: 0.0351.. Train accuracy: 0.9878.. Top-3 train accuracy: 0.9996.. Test loss: 0.4196.. Test accuracy: 0.9122.. Top-3 test accuracy: 0.9885\n",
      "Epoch 77/100.. Time per epoch: 178.2992.. Average time per step: 0.4560.. Train loss: 0.0355.. Train accuracy: 0.9872.. Top-3 train accuracy: 0.9998.. Test loss: 0.4281.. Test accuracy: 0.9093.. Top-3 test accuracy: 0.9849\n",
      "Epoch 78/100.. Time per epoch: 178.3364.. Average time per step: 0.4561.. Train loss: 0.0351.. Train accuracy: 0.9880.. Top-3 train accuracy: 0.9998.. Test loss: 0.4033.. Test accuracy: 0.9137.. Top-3 test accuracy: 0.9868\n",
      "Epoch 79/100.. Time per epoch: 178.0594.. Average time per step: 0.4554.. Train loss: 0.0329.. Train accuracy: 0.9885.. Top-3 train accuracy: 0.9998.. Test loss: 0.4221.. Test accuracy: 0.9116.. Top-3 test accuracy: 0.9882\n",
      "Epoch 80/100.. Time per epoch: 178.1898.. Average time per step: 0.4557.. Train loss: 0.0368.. Train accuracy: 0.9869.. Top-3 train accuracy: 0.9998.. Test loss: 0.4499.. Test accuracy: 0.9083.. Top-3 test accuracy: 0.9864\n",
      "Epoch 81/100.. Time per epoch: 178.0795.. Average time per step: 0.4554.. Train loss: 0.0292.. Train accuracy: 0.9904.. Top-3 train accuracy: 0.9998.. Test loss: 0.4273.. Test accuracy: 0.9131.. Top-3 test accuracy: 0.9856\n",
      "Epoch 82/100.. Time per epoch: 178.4979.. Average time per step: 0.4565.. Train loss: 0.0368.. Train accuracy: 0.9873.. Top-3 train accuracy: 0.9997.. Test loss: 0.4303.. Test accuracy: 0.9118.. Top-3 test accuracy: 0.9874\n",
      "Epoch 83/100.. Time per epoch: 178.3666.. Average time per step: 0.4562.. Train loss: 0.0345.. Train accuracy: 0.9878.. Top-3 train accuracy: 0.9998.. Test loss: 0.4385.. Test accuracy: 0.9086.. Top-3 test accuracy: 0.9862\n",
      "Epoch 84/100.. Time per epoch: 178.4624.. Average time per step: 0.4564.. Train loss: 0.0324.. Train accuracy: 0.9886.. Top-3 train accuracy: 0.9998.. Test loss: 0.4514.. Test accuracy: 0.9119.. Top-3 test accuracy: 0.9864\n",
      "Epoch 85/100.. Time per epoch: 177.9460.. Average time per step: 0.4551.. Train loss: 0.0306.. Train accuracy: 0.9896.. Top-3 train accuracy: 0.9998.. Test loss: 0.4656.. Test accuracy: 0.9080.. Top-3 test accuracy: 0.9849\n",
      "Epoch 86/100.. Time per epoch: 177.8748.. Average time per step: 0.4549.. Train loss: 0.0368.. Train accuracy: 0.9874.. Top-3 train accuracy: 0.9997.. Test loss: 0.4438.. Test accuracy: 0.9093.. Top-3 test accuracy: 0.9866\n",
      "Epoch 87/100.. Time per epoch: 177.3661.. Average time per step: 0.4536.. Train loss: 0.0309.. Train accuracy: 0.9896.. Top-3 train accuracy: 0.9999.. Test loss: 0.4282.. Test accuracy: 0.9123.. Top-3 test accuracy: 0.9865\n",
      "Epoch 88/100.. Time per epoch: 177.6966.. Average time per step: 0.4545.. Train loss: 0.0302.. Train accuracy: 0.9891.. Top-3 train accuracy: 0.9996.. Test loss: 0.4180.. Test accuracy: 0.9126.. Top-3 test accuracy: 0.9858\n",
      "Epoch 89/100.. Time per epoch: 177.7571.. Average time per step: 0.4546.. Train loss: 0.0299.. Train accuracy: 0.9895.. Top-3 train accuracy: 0.9998.. Test loss: 0.4353.. Test accuracy: 0.9125.. Top-3 test accuracy: 0.9874\n",
      "Epoch 90/100.. Time per epoch: 177.7932.. Average time per step: 0.4547.. Train loss: 0.0309.. Train accuracy: 0.9899.. Top-3 train accuracy: 0.9998.. Test loss: 0.4339.. Test accuracy: 0.9120.. Top-3 test accuracy: 0.9870\n",
      "Epoch 91/100.. Time per epoch: 178.1237.. Average time per step: 0.4556.. Train loss: 0.0317.. Train accuracy: 0.9890.. Top-3 train accuracy: 0.9997.. Test loss: 0.4305.. Test accuracy: 0.9137.. Top-3 test accuracy: 0.9860\n",
      "Epoch 92/100.. Time per epoch: 178.0791.. Average time per step: 0.4554.. Train loss: 0.0269.. Train accuracy: 0.9906.. Top-3 train accuracy: 0.9998.. Test loss: 0.4556.. Test accuracy: 0.9131.. Top-3 test accuracy: 0.9866\n",
      "Epoch 93/100.. Time per epoch: 177.6530.. Average time per step: 0.4544.. Train loss: 0.0332.. Train accuracy: 0.9885.. Top-3 train accuracy: 0.9996.. Test loss: 0.4293.. Test accuracy: 0.9129.. Top-3 test accuracy: 0.9875\n",
      "Epoch 94/100.. Time per epoch: 177.5330.. Average time per step: 0.4540.. Train loss: 0.0296.. Train accuracy: 0.9893.. Top-3 train accuracy: 0.9999.. Test loss: 0.4147.. Test accuracy: 0.9148.. Top-3 test accuracy: 0.9860\n",
      "Epoch 95/100.. Time per epoch: 177.1752.. Average time per step: 0.4531.. Train loss: 0.0273.. Train accuracy: 0.9905.. Top-3 train accuracy: 0.9998.. Test loss: 0.4581.. Test accuracy: 0.9117.. Top-3 test accuracy: 0.9855\n",
      "Epoch 96/100.. Time per epoch: 177.4039.. Average time per step: 0.4537.. Train loss: 0.0260.. Train accuracy: 0.9911.. Top-3 train accuracy: 0.9998.. Test loss: 0.4646.. Test accuracy: 0.9109.. Top-3 test accuracy: 0.9858\n",
      "Epoch 97/100.. Time per epoch: 177.0641.. Average time per step: 0.4528.. Train loss: 0.0272.. Train accuracy: 0.9905.. Top-3 train accuracy: 0.9999.. Test loss: 0.4489.. Test accuracy: 0.9126.. Top-3 test accuracy: 0.9866\n",
      "Epoch 98/100.. Time per epoch: 177.2225.. Average time per step: 0.4533.. Train loss: 0.0253.. Train accuracy: 0.9913.. Top-3 train accuracy: 0.9999.. Test loss: 0.4705.. Test accuracy: 0.9111.. Top-3 test accuracy: 0.9864\n",
      "Epoch 99/100.. Time per epoch: 176.8577.. Average time per step: 0.4523.. Train loss: 0.0255.. Train accuracy: 0.9906.. Top-3 train accuracy: 0.9999.. Test loss: 0.4669.. Test accuracy: 0.9110.. Top-3 test accuracy: 0.9849\n",
      "Epoch 100/100.. Time per epoch: 177.0534.. Average time per step: 0.4528.. Train loss: 0.0280.. Train accuracy: 0.9905.. Top-3 train accuracy: 0.9999.. Test loss: 0.4613.. Test accuracy: 0.9083.. Top-3 test accuracy: 0.9854\n"
     ]
    }
   ],
   "source": [
    "#train the model\n",
    "model.to(device)\n",
    "\n",
    "steps = 0\n",
    "running_loss = 0\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    since = time.time()\n",
    "    \n",
    "    train_accuracy = 0\n",
    "    top3_train_accuracy = 0 \n",
    "    for inputs, labels in trainloader:\n",
    "        steps += 1\n",
    "        # Move input and label tensors to the default device\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        logps = model.forward(inputs)\n",
    "        loss = criterion(logps, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        # calculate train top-1 accuracy\n",
    "        ps = torch.exp(logps)\n",
    "        top_p, top_class = ps.topk(1, dim=1)\n",
    "        equals = top_class == labels.view(*top_class.shape)\n",
    "        train_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "        \n",
    "        # Calculate train top-3 accuracy\n",
    "        np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "        target_numpy = labels.cpu().numpy()\n",
    "        top3_train_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "        \n",
    "    time_elapsed = time.time() - since\n",
    "    \n",
    "    test_loss = 0\n",
    "    test_accuracy = 0\n",
    "    top3_test_accuracy = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in testloader:\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            logps = model.forward(inputs)\n",
    "            batch_loss = criterion(logps, labels)\n",
    "\n",
    "            test_loss += batch_loss.item()\n",
    "\n",
    "            # Calculate test top-1 accuracy\n",
    "            ps = torch.exp(logps)\n",
    "            top_p, top_class = ps.topk(1, dim=1)\n",
    "            equals = top_class == labels.view(*top_class.shape)\n",
    "            test_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "            \n",
    "            # Calculate test top-3 accuracy\n",
    "            np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "            target_numpy = labels.cpu().numpy()\n",
    "            top3_test_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "\n",
    "    print(f\"Epoch {epoch+1}/{epochs}.. \"\n",
    "          f\"Time per epoch: {time_elapsed:.4f}.. \"\n",
    "          f\"Average time per step: {time_elapsed/len(trainloader):.4f}.. \"\n",
    "          f\"Train loss: {running_loss/len(trainloader):.4f}.. \"\n",
    "          f\"Train accuracy: {train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Top-3 train accuracy: {top3_train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Test loss: {test_loss/len(testloader):.4f}.. \"\n",
    "          f\"Test accuracy: {test_accuracy/len(testloader):.4f}.. \"\n",
    "          f\"Top-3 test accuracy: {top3_test_accuracy/len(testloader):.4f}\")\n",
    "\n",
    "    train_stats = train_stats.append({'Epoch': epoch, 'Time per epoch':time_elapsed, 'Avg time per step': time_elapsed/len(trainloader), 'Train loss' : running_loss/len(trainloader), 'Train accuracy': train_accuracy/len(trainloader), 'Train top-3 accuracy':top3_train_accuracy/len(trainloader),'Test loss' : test_loss/len(testloader), 'Test accuracy': test_accuracy/len(testloader), 'Test top-3 accuracy':top3_test_accuracy/len(testloader)}, ignore_index=True)\n",
    "\n",
    "    running_loss = 0\n",
    "    model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats.to_csv('train_log_ResNext50_ReLU.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
